"""
Case Type   : 极致RTO-数据库系统类
Case Name   : 开启极致RTO，备机进行switchover过程中，集群gs_om stop后恢复
Create At   : 2020/10/15
Owner       : @peilinqian
Description :
    1、开启极致RTO
    2、备机进行switchover
    3、数据库gs_om -t stop
    4、switchover失败，重启主备；或者switchover成功，重启主备，备机进行重建；
    5、备机再次进行switchover
    6、新主机创建数据表，并插入数据功能正常
    7、还原主备，原主机进行switchover
    8、还原主机创建数据表，并插入数据功能正常
    9、执行完成后，关闭极致RTO
    10、查询主备数据是否一致
    11、创建数据表，并插入数据功能正常
Expect      :
    1、开启极致RTO成功
    2、备机开始进行switchover
    3、数据库gs_om -t stop成功
    4、switchover失败，重启主备成功；或者switchover成功，重启主备，备机进行重建成功；
    5、备机再次进行switchover成功
    6、新主机创建数据表，并插入数据功能正常
    7、还原主备，原主机进行switchover成功
    8、还原主机创建数据表，并插入数据功能正常
    9、执行完成后，关闭极致RTO成功
    10、查询主备数据一致
    11、创建数据表，并插入数据功能正常
History     :
    modified：2021-10-9 by 5328113;switchover过程中，进行om stop集群，结果可能成功，
        增加swithover成功处理流程
    modified：2021-3-9 by 5328113;主备同步方法调用修改，增加超时等待时间；
    modified：2021-2-5 by 5328113;guc参数还原时指定还原值
    modified ：2020-11-18 by @peilinqian ；完善TPCC断言；
    modified ：2020-12-14 by @peilinqian ;teardown增加主备异常恢复；tpcc执行完成后增加同步校验
    modified ：2021-1-13 by @peilinqian ；
        进行switchover前增加主备同步校验；删除kill-9 断言；删除不必要tpcc操作
    modified:2021-1-28 by 5328113;删除start成功断言，删除不必要tpcc
    modified ：2024-7-10 by @peilinqian ；增加switchover成功与失败异常场景处理
"""

import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node
from yat.test import macro


class ExtremeRto(unittest.TestCase):
    nodes_tuple = ('PrimaryDbUser', 'Standby1DbUser', 'Standby2DbUser')

    @RestartDbCluster(*nodes_tuple)
    def setUp(self):
        self.log = Logger()
        self.log.info(
            '----Opengauss_Reliability_RTO_Dbsys_Case026:初始化----')

        self.pri_root = Node(node='PrimaryRoot')
        self.pri_dbuser = Node(node='PrimaryDbUser')
        self.sta1_root = Node(node='Standby1Root')
        self.sta1_dbuser = Node(node='Standby1DbUser')
        self.sta2_dbuser = Node(node='Standby2DbUser')
        self.pri_sh = CommonSH('PrimaryDbUser')
        self.sta_1_sh = CommonSH('Standby1DbUser')
        self.sta_2_sh = CommonSH('Standby2DbUser')
        self.com = Common()
        self.constant = Constant()
        self.tpcc_path = macro.TPCC_PATH
        self.MAX_NUM = 10000
        self.table_test = 't_RTO_Dbsys_Case026'
        self.insert_sql = f'''drop table if exists {self.table_test};
            create table {self.table_test}(id int);
            begin 
            for i in 1..{self.MAX_NUM} loop
            insert into {self.table_test} values(i);
            end loop;
            end;'''
        self.select_sql = f'select count(*) from {self.table_test};'
        self.drop_sql = f'drop table {self.table_test};'
        self.node_num = self.com.get_node_num(self.pri_root)
        self.log.info(self.node_num)
        self.max_time = 1200
        self.min_time = 300

    def test_main(self):
        try:
            self.log.info('----1、开启极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'replication_type=1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers=4')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=off')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            self.log.info('----查看主备是否同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----2、备机进行switchover----')
            switch_thread = ComThread(self.sta_1_sh.executeGsctl,
                                      args=('switchover',
                                            self.constant.SWITCH_SUCCESS_MSG,
                                            '', False))
            switch_thread.setDaemon(True)
            switch_thread.start()

            self.log.info('----3、数据库gs_om -t stop----')
            stop_msg = self.pri_sh.stopDbCluster()
            self.assertTrue(stop_msg)

            self.log.info('----等待switchover完成----')
            switch_thread.join()
            switch_result = switch_thread.get_result()
            self.log.info(switch_result)

            self.log.info('----4、switchover后，重启主备----')
            self.pri_sh.startDbCluster()
            if not switch_result:
                self.log.info('----switchover失败场景----')
                self.log.info('----等待主备同步----')
                self.pri_sh.check_location_consistency('primary',
                                                       self.node_num,
                                                       self.min_time)
                self.log.info('----查询主备状态,不正常通过build可恢复----')
                status = self.pri_sh.getDbClusterStatus('status')
                if not status:
                    self.log.info('----build备机----')
                    success_msg = self.constant.REBUILD_SUCCESS_MSG
                    build_result = self.sta_1_sh.executeGsctl('build',
                                                              success_msg)
                    self.assertTrue(build_result)
            else:
                self.log.info('----switchover成功场景----')
                self.log.info('----查询主备状态；expect：状态异常----')
                status = self.pri_sh.getDbClusterStatus('status')
                self.assertFalse(status)
                self.log.info('----build备机；expect：成功----')
                success_msg = self.constant.REBUILD_SUCCESS_MSG
                build_result = self.sta_1_sh.executeGsctl('build',
                                                          success_msg)
                self.assertTrue(build_result)
                build_result = self.sta_2_sh.executeGsctl('build',
                                                          success_msg)
                self.assertTrue(build_result)
            self.log.info('----等待主备同步----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)
            self.log.info('----查询主备状态；expect：成功----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----5、备机再次进行switchover----')
            success_msg = self.constant.SWITCH_SUCCESS_MSG
            switch_result = self.sta_1_sh.executeGsctl('switchover',
                                                       success_msg)
            self.assertTrue(switch_result)

            self.log.info('----进行refresh----')
            refresh_msg = self.sta_1_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.log.info('----查看主机query，同步是否正常----')
            self.sta_1_sh.check_location_consistency('primary',
                                                     self.node_num,
                                                     self.min_time)

            self.log.info('----查询主备状态----')
            self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----6、新主机创建数据表，并插入数据功能正常----')
            insert_result = self.sta_1_sh.executDbSql(self.insert_sql)
            self.log.info(insert_result)
            select_result = self.sta_1_sh.executDbSql(self.select_sql)
            self.log.info(select_result)
            self.assertIn(str(self.MAX_NUM), select_result)

            self.log.info('----查看主机query，同步是否正常----')
            self.sta_1_sh.check_location_consistency('primary',
                                                     self.node_num,
                                                     self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----7、还原主备，原主机进行switchover----')
            success_msg = self.constant.SWITCH_SUCCESS_MSG
            switch_result = self.pri_sh.executeGsctl('switchover',
                                                     success_msg)
            self.assertTrue(switch_result)
            self.log.info('----进行refresh----')
            refresh_msg = self.pri_sh.exec_refresh_conf()
            self.assertTrue(refresh_msg)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

            self.log.info('----创建数据表，并插入数据功能正常----')
            insert_result = self.pri_sh.executDbSql(self.insert_sql)
            self.log.info(insert_result)
            select_result = self.pri_sh.executDbSql(self.select_sql)
            self.log.info(select_result)
            self.assertIn(str(self.MAX_NUM), select_result)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

        finally:
            self.log.info('----9、执行完成后，关闭极致RTO----')
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_parse_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'recovery_redo_workers = 1')
            self.assertTrue(msg)
            msg = self.pri_sh.executeGsguc('set',
                                           self.constant.GSGUC_SUCCESS_MSG,
                                           'hot_standby=on')
            self.assertTrue(msg)
            # 重启数据库配置文件生效
            self.pri_sh.stopDbCluster()
            time.sleep(2)
            self.pri_sh.startDbCluster()

            # 校验备机是否完成数据同步
            self.log.info('校验备机是否完成数据同步')
            flag_1 = self.sta_1_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_1)
            flag_2 = self.sta_2_sh.check_location_consistency('standby',
                                                              self.node_num,
                                                              self.min_time)
            self.assertTrue(flag_2)

            self.log.info('----10、查询主备数据是否一致----')
            nodes_tuple = (self.pri_dbuser,
                           self.sta1_dbuser,
                           self.sta2_dbuser)
            flag = self.com.check_data_sample_by_all(r'\d', *nodes_tuple)
            self.assertTrue(flag)

            table_dict = self.com.format_sql_result(
                self.pri_sh.executDbSql(r'\d'))
            table_name = table_dict.get('Name')
            for name in table_name:
                select_sql = f'select count(*) from {name};'
                flag = self.com.check_data_sample_by_all(select_sql,
                                                         *nodes_tuple)
                self.assertTrue(flag)

            self.log.info('----11、创建数据表，并插入数据功能正常----')
            insert_result = self.pri_sh.executDbSql(self.insert_sql)
            self.log.info(insert_result)
            select_result = self.pri_sh.executDbSql(self.select_sql)
            self.log.info(select_result)
            self.assertIn(str(self.MAX_NUM), select_result)

            self.log.info('----查看主机query，同步是否正常----')
            self.pri_sh.check_location_consistency('primary',
                                                   self.node_num,
                                                   self.min_time)

            self.log.info('----查询主备状态----')
            status = self.pri_sh.getDbClusterStatus('status')
            self.assertTrue(status)

    def tearDown(self):
        self.log.info('----恢复主备关系-----------------')
        host_name = self.pri_root.sh('hostname').result()
        result = self.pri_sh.check_whether_need_switch(host_name)
        if result:
            success_msg = self.constant.SWITCHOVER_SUCCESS_MSG
            result = self.pri_sh.executeGsctl('switchover', success_msg)
            self.log.info(result)
            result = self.pri_sh.exec_refresh_conf()
            self.log.info(result)

        self.log.info('----清理测试表----')
        drop_result = self.pri_sh.executDbSql(self.drop_sql)
        self.log.info(drop_result)

        self.log.info('----校验刷脏页是否完成----')
        self.pri_sh.check_whether_remain_dirty_page()

        self.log.info(
            '----Opengauss_Reliability_RTO_Dbsys_Case026:执行完毕----')
